{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Sat Sep  9 14:13:37 2017\n",
    "\n",
    "@author: stevewyl\n",
    "\"\"\"\n",
    "# 保证映射后结构一样\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "# 文本预处理\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "# 将类别映射成需要的格式\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "# 这个是连接层\n",
    "from keras.layers.merge import concatenate\n",
    "\n",
    "# 搭建模型\n",
    "from keras.models import Sequential, Model\n",
    "\n",
    "# 这个是层的搭建\n",
    "from keras.layers import Dense, Embedding, Activation, Input\n",
    "\n",
    "from keras.layers import Convolution1D, Flatten, Dropout, MaxPool1D\n",
    "\n",
    "from keras.layers import  BatchNormalization\n",
    "from keras.layers import Conv1D,MaxPooling1D\n",
    "\n",
    "# 数据分割\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 数据管道\n",
    "from sklearn.pipeline import Pipeline,make_pipeline\n",
    "# 数据处理\n",
    "from data_helper_ml import load_data_and_labels\n",
    "\n",
    "# 数据可视化\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "good_data_file = \"./data/good_cut_jieba.txt\"\n",
    "bad_data_file = \"./data/bad_cut_jieba.txt\"\n",
    "mid_data_file = \"./data/mid_cut_jieba.txt\"\n",
    "x_text, y = load_data_and_labels(good_data_file, bad_data_file, mid_data_file)\n",
    "\n",
    "# Tokenizer是一个用于向量化文本，或将文本转换为序列\n",
    "tokenizer = Tokenizer(filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',lower=True,split=\" \")\n",
    "tokenizer.fit_on_texts(x_text)\n",
    "vocab = tokenizer.word_index\n",
    "\n",
    "# 数据分割\n",
    "x_train, x_test, y_train, y_test = train_test_split(x_text, y, test_size=0.2, random_state=2017)\n",
    "#映射成数字\n",
    "x_train_word_ids = tokenizer.texts_to_sequences(x_train)\n",
    "x_test_word_ids = tokenizer.texts_to_sequences(x_test)\n",
    "#让他共同化\n",
    "x_train_padded_seqs = pad_sequences(x_train_word_ids, maxlen=64)\n",
    "x_test_padded_seqs = pad_sequences(x_test_word_ids, maxlen=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15648\n",
      "15648\n"
     ]
    }
   ],
   "source": [
    "print(len(x_text))\n",
    "print(len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 64, 300)           4594500   \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 64, 256)           384256    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 22, 256)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 22, 128)           163968    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 8, 128)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 8, 64)             24640     \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 3)                 771       \n",
      "=================================================================\n",
      "Total params: 5,301,511\n",
      "Trainable params: 5,300,487\n",
      "Non-trainable params: 1,024\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(len(vocab) + 1, 300, input_length=64)) #使用Embeeding层将每个词编码转换为词向量\n",
    "model.add(Conv1D(256, 5, padding='same'))\n",
    "model.add(MaxPooling1D(3, 3, padding='same'))\n",
    "model.add(Conv1D(128, 5, padding='same'))\n",
    "model.add(MaxPooling1D(3, 3, padding='same'))\n",
    "model.add(Conv1D(64, 3, padding='same'))\n",
    "model.add(Flatten())\n",
    "model.add(Dropout(0.1))\n",
    "model.add(BatchNormalization())  # (批)规范化层\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "  800/12518 [>.............................] - ETA: 1:05 - loss: 1.1991 - accuracy: 0.3688"
     ]
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels,epochs=5, batch_size=800)\n",
    "y_predict = model.predict_classes(x_test_padded_seqs)  # 预测的是类别，结果就是类别号\n",
    "y_predict = list(map(str, y_predict))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score,f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print('准确率', accuracy_score(y_test, y_predict))\n",
    "# print('平均f1-score:',f1_score(y_test, y_predict, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# textCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 64)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 64, 300)      4594500     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, 64, 256)      230656      embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, 64, 256)      307456      embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_9 (Conv1D)               (None, 64, 256)      384256      embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1D)  (None, 1, 256)       0           conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1D)  (None, 1, 256)       0           conv1d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_7 (MaxPooling1D)  (None, 1, 256)       0           conv1d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 1, 768)       0           max_pooling1d_5[0][0]            \n",
      "                                                                 max_pooling1d_6[0][0]            \n",
      "                                                                 max_pooling1d_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 768)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 768)          0           flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 3)            2307        dropout_5[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 5,519,175\n",
      "Trainable params: 924,675\n",
      "Non-trainable params: 4,594,500\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "main_input = Input(shape=(64,), dtype='float64')\n",
    "# 词嵌入（使用预训练的词向量）\n",
    "embedder = Embedding(len(vocab) + 1, 300, input_length=64, trainable=False)\n",
    "\n",
    "embed = embedder(main_input)\n",
    "\n",
    "# 词窗大小分别为3,4,5\n",
    "cnn1 = Conv1D(256, 3, padding='same', strides=1, activation='relu')(embed)\n",
    "\n",
    "cnn1 = MaxPooling1D(pool_size=48)(cnn1)\n",
    "\n",
    "cnn2 = Conv1D(256, 4, padding='same', strides=1, activation='relu')(embed)\n",
    "\n",
    "cnn2 = MaxPooling1D(pool_size=47)(cnn2)\n",
    "\n",
    "cnn3 = Conv1D(256, 5, padding='same', strides=1, activation='relu')(embed)\n",
    "\n",
    "cnn3 = MaxPooling1D(pool_size=46)(cnn3)\n",
    "\n",
    "# 合并三个模型的输出向量\n",
    "cnn = concatenate([cnn1, cnn2, cnn3], axis=-1)\n",
    "\n",
    "flat = Flatten()(cnn)\n",
    "\n",
    "drop = Dropout(0.2)(flat)\n",
    "\n",
    "main_output = Dense(3, activation='softmax')(drop)\n",
    "\n",
    "model = Model(inputs=main_input, outputs=main_output)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12518/12518 [==============================] - 47s 4ms/step - loss: 1.0583 - accuracy: 0.4372\n",
      "Epoch 2/10\n",
      "12518/12518 [==============================] - 48s 4ms/step - loss: 1.0242 - accuracy: 0.4754\n",
      "Epoch 3/10\n",
      "12518/12518 [==============================] - 50s 4ms/step - loss: 1.0010 - accuracy: 0.5000\n",
      "Epoch 4/10\n",
      "12518/12518 [==============================] - 52s 4ms/step - loss: 0.9741 - accuracy: 0.5218\n",
      "Epoch 5/10\n",
      "12518/12518 [==============================] - 48s 4ms/step - loss: 0.9466 - accuracy: 0.5429\n",
      "Epoch 6/10\n",
      "12518/12518 [==============================] - 50s 4ms/step - loss: 0.9223 - accuracy: 0.5582\n",
      "Epoch 7/10\n",
      "12518/12518 [==============================] - 50s 4ms/step - loss: 0.8992 - accuracy: 0.5678\n",
      "Epoch 8/10\n",
      "12518/12518 [==============================] - 48s 4ms/step - loss: 0.8768 - accuracy: 0.5848\n",
      "Epoch 9/10\n",
      "12518/12518 [==============================] - 47s 4ms/step - loss: 0.8555 - accuracy: 0.5971\n",
      "Epoch 10/10\n",
      "12518/12518 [==============================] - 48s 4ms/step - loss: 0.8345 - accuracy: 0.6092\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1363e9135f8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "#y_test_onehot = keras.utils.to_categorical(y_test, num_classes=3)  # 将标签转换为one-hot编码\n",
    "result = model.predict(x_test_padded_seqs)  # 预测样本属于每个类别的概率\n",
    "\n",
    "result_labels = np.argmax(result, axis=1)  # 获得最大概率对应的标签\n",
    "\n",
    "y_predict = list(map(str, result_labels))\n",
    "# print(y_predict)\n",
    "# print('准确率', metrics.accuracy_score(y_test, y_predict))\n",
    "\n",
    "# print('平均f1-score:', metrics.f1_score(y_test, y_predict, average='weighted'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用Word2Vec词向量的TextCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v_model=Word2Vec.load('sentiment_analysis/w2v_model.pkl')\n",
    "# 预训练的词向量中没有出现的词用0向量表示\n",
    "embedding_matrix = np.zeros((len(vocab) + 1, 300))\n",
    "for word, i in vocab.items():\n",
    "    try:\n",
    "        embedding_vector = w2v_model[str(word)]\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "    except KeyError:\n",
    "        continue\n",
    "        \n",
    " #构建TextCNN模型\n",
    "def TextCNN_model_2(x_train_padded_seqs,y_train,x_test_padded_seqs,y_test,embedding_matrix):\n",
    "    # 模型结构：词嵌入-卷积池化*3-拼接-全连接-dropout-全连接\n",
    "    main_input = Input(shape=(50,), dtype='float64')\n",
    "    # 词嵌入（使用预训练的词向量）\n",
    "    embedder = Embedding(len(vocab) + 1, 300, input_length=50, weights=[embedding_matrix], trainable=False)\n",
    "    #embedder = Embedding(len(vocab) + 1, 300, input_length=50, trainable=False)\n",
    "    embed = embedder(main_input)\n",
    "    # 词窗大小分别为3,4,5\n",
    "    cnn1 = Conv1D(256, 3, padding='same', strides=1, activation='relu')(embed)\n",
    "    cnn1 = MaxPooling1D(pool_size=38)(cnn1)\n",
    "    cnn2 = Conv1D(256, 4, padding='same', strides=1, activation='relu')(embed)\n",
    "    cnn2 = MaxPooling1D(pool_size=37)(cnn2)\n",
    "    cnn3 = Conv1D(256, 5, padding='same', strides=1, activation='relu')(embed)\n",
    "    cnn3 = MaxPooling1D(pool_size=36)(cnn3)\n",
    "    # 合并三个模型的输出向量\n",
    "    cnn = concatenate([cnn1, cnn2, cnn3], axis=-1)\n",
    "    flat = Flatten()(cnn)\n",
    "    drop = Dropout(0.2)(flat)\n",
    "    main_output = Dense(3, activation='softmax')(drop)\n",
    "    model = Model(inputs=main_input, outputs=main_output)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    " \n",
    "    one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "    model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=20)\n",
    "    #y_test_onehot = keras.utils.to_categorical(y_test, num_classes=3)  # 将标签转换为one-hot编码\n",
    "    result = model.predict(x_test_padded_seqs)  # 预测样本属于每个类别的概率\n",
    "    result_labels = np.argmax(result, axis=1)  # 获得最大概率对应的标签\n",
    "    y_predict = list(map(str, result_labels))\n",
    "    print('准确率', metrics.accuracy_score(y_test, y_predict))\n",
    "    print('平均f1-score:', metrics.f1_score(y_test, y_predict, average='weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# 导入使用到的库\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Embedding, Activation, merge, Input, Lambda, Reshape\n",
    "from keras.layers import Convolution1D, Flatten, Dropout, MaxPool1D, GlobalAveragePooling1D\n",
    "from keras.layers import LSTM, GRU, TimeDistributed, Bidirectional\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras import initializers\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One-Hot + MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# model = Sequential()\n",
    "# # 全连接层\n",
    "# model.add(Dense(512, input_shape=(len(vocab)+1,), activation='relu'))\n",
    "# # DropOut层\n",
    "# model.add(Dropout(0.5))\n",
    "# # 全连接层+分类器\n",
    "# model.add(Dense(3,activation='softmax'))\n",
    " \n",
    "# model.compile(loss='categorical_crossentropy',\n",
    "#               optimizer='adam',\n",
    "#               metrics=['accuracy'])\n",
    "# #  validation_data=(x_test, y_test)\n",
    "# model.fit(x_train, y_train,\n",
    "#           batch_size=32,\n",
    "#           epochs=15\n",
    "#           )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_7 (Embedding)      (None, 64, 300)           4594500   \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 256)               570368    \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 3)                 771       \n",
      "=================================================================\n",
      "Total params: 5,165,639\n",
      "Trainable params: 5,165,639\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 模型结构：词嵌入-LSTM-全连接\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(vocab)+1, 300, input_length=64))\n",
    "model.add(LSTM(256, dropout=0.2, recurrent_dropout=0.1))\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12518/12518 [==============================] - 80s 6ms/step - loss: 1.0250 - accuracy: 0.4867\n",
      "Epoch 2/10\n",
      "12518/12518 [==============================] - 74s 6ms/step - loss: 0.7618 - accuracy: 0.6507\n",
      "Epoch 3/10\n",
      "12518/12518 [==============================] - 75s 6ms/step - loss: 0.5988 - accuracy: 0.7440\n",
      "Epoch 4/10\n",
      "12518/12518 [==============================] - 74s 6ms/step - loss: 0.5064 - accuracy: 0.7956\n",
      "Epoch 5/10\n",
      "12518/12518 [==============================] - 78s 6ms/step - loss: 0.4221 - accuracy: 0.8364\n",
      "Epoch 6/10\n",
      "12518/12518 [==============================] - 74s 6ms/step - loss: 0.3631 - accuracy: 0.8647\n",
      "Epoch 7/10\n",
      "12518/12518 [==============================] - 74s 6ms/step - loss: 0.3184 - accuracy: 0.8824\n",
      "Epoch 8/10\n",
      "12518/12518 [==============================] - 76s 6ms/step - loss: 0.2824 - accuracy: 0.8958\n",
      "Epoch 9/10\n",
      "12518/12518 [==============================] - 74s 6ms/step - loss: 0.2551 - accuracy: 0.9065\n",
      "Epoch 10/10\n",
      "12518/12518 [==============================] - 80s 6ms/step - loss: 0.2302 - accuracy: 0.9147\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x136473807b8>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_7 (Embedding)      (None, 64, 300)           4594500   \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 256)               570368    \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 3)                 771       \n",
      "=================================================================\n",
      "Total params: 5,165,639\n",
      "Trainable params: 5,165,639\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bi-GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 64, 300)           4594500   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 64, 512)           855552    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 512)               1181184   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3)                 1539      \n",
      "=================================================================\n",
      "Total params: 6,632,775\n",
      "Trainable params: 6,632,775\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 模型结构：词嵌入-双向GRU*2-全连接\n",
    "model = Sequential()\n",
    "# 64是序列号\n",
    "model.add(Embedding(len(vocab)+1, 300, input_length=64))\n",
    "model.add(Bidirectional(GRU(256, dropout=0.2, recurrent_dropout=0.1, return_sequences=True)))\n",
    "model.add(Bidirectional(GRU(256, dropout=0.2, recurrent_dropout=0.1)))\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " 4800/12518 [==========>...................] - ETA: 5:04 - loss: 1.0801 - accuracy: 0.3985- ETA: 7:49 - loss: 1.0938 - accuracy: "
     ]
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN+RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "C-LSTM串联（将CNN的输出直接拼接上RNN）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 模型结构：词嵌入-卷积池化-GRU*2-全连接\n",
    "model = Sequential()\n",
    "model.add(Embedding(len(vocab)+1, 300, input_length=64))\n",
    "model.add(Convolution1D(256, 3, padding='same', strides = 1))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPool1D(pool_size=2))\n",
    "model.add(GRU(256, dropout=0.2, recurrent_dropout=0.1, return_sequences = True))\n",
    "model.add(GRU(256, dropout=0.2, recurrent_dropout=0.1))\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# CNN+RNN并联"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 模型结构：词嵌入-卷积池化-全连接 ---拼接-全连接\n",
    "#                -双向GRU-全连接\n",
    "main_input = Input(shape=(20,), dtype='float64')\n",
    "embed = Embedding(len(vocab)+1, 300, input_length=64)(main_input)\n",
    "cnn = Convolution1D(256, 3, padding='same', strides = 1, activation='relu')(embed)\n",
    "cnn = MaxPool1D(pool_size=4)(cnn)\n",
    "cnn = Flatten()(cnn)\n",
    "cnn = Dense(256)(cnn)\n",
    "rnn = Bidirectional(GRU(256, dropout=0.2, recurrent_dropout=0.1))(embed)\n",
    "rnn = Dense(256)(rnn)\n",
    "con = concatenate([cnn,rnn], axis=-1)\n",
    "main_output = Dense(3, activation='softmax')(con)\n",
    "model = Model(inputs = main_input, outputs = main_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# RCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # 模型结构：词嵌入*3-LSTM*2-拼接-全连接-最大化池化-全连接\n",
    "# # 我们需要重新整理数据集\n",
    "# left_train_word_ids = [[len(vocab)] + x[:-1] for x in x_train_word_ids]\n",
    "# left_test_word_ids = [[len(vocab)] + x[:-1] for x in x_test_word_ids]\n",
    "# right_train_word_ids = [x[1:] + [len(vocab)] for x in x_train_word_ids]\n",
    "# right_test_word_ids = [x[1:] + [len(vocab)] for x in x_test_word_ids]\n",
    " \n",
    "# # 分别对左边和右边的词进行编码\n",
    "# left_train_padded_seqs = pad_sequences(left_train_word_ids, maxlen=20)\n",
    "# left_test_padded_seqs = pad_sequences(left_test_word_ids, maxlen=20)\n",
    "# right_train_padded_seqs = pad_sequences(right_train_word_ids, maxlen=20)\n",
    "# right_test_padded_seqs = pad_sequences(right_test_word_ids, maxlen=20)\n",
    " \n",
    "# # 模型共有三个输入，分别是左词，右词和中心词\n",
    "# document = Input(shape = (None, ), dtype = \"int32\")\n",
    "# left_context = Input(shape = (None, ), dtype = \"int32\")\n",
    "# right_context = Input(shape = (None, ), dtype = \"int32\")\n",
    " \n",
    "# # 构建词向量\n",
    "# embedder = Embedding(len(vocab) + 1, 300, input_length = 64)\n",
    "# doc_embedding = embedder(document)\n",
    "# l_embedding = embedder(left_context)\n",
    "# r_embedding = embedder(right_context)\n",
    " \n",
    "# # 分别对应文中的公式(1)-(7)\n",
    "# forward = LSTM(256, return_sequences = True)(l_embedding) # 等式(1)\n",
    "# # 等式(2)\n",
    "# backward = LSTM(256, return_sequences = True, go_backwards = True)(r_embedding) \n",
    "# together = concatenate([forward, doc_embedding, backward], axis = 2) # 等式(3)\n",
    " \n",
    "# semantic = TimeDistributed(Dense(128, activation = \"tanh\"))(together) # 等式(4)\n",
    "# # 等式(5)\n",
    "# pool_rnn = Lambda(lambda x: backend.max(x, axis = 1), output_shape = (128, ))(semantic) \n",
    "# output = Dense(3, activation = \"softmax\")(pool_rnn) # 等式(6)和(7)\n",
    "# model = Model(inputs = [document, left_context, right_context], outputs = output)\n",
    "# model.summary()\n",
    "# model.compile(loss='categorical_crossentropy',\n",
    "#               optimizer='adam',\n",
    "#               metrics=['accuracy'])\n",
    " \n",
    "# model.fit([X_train_padded_seqs, left_train_padded_seqs, right_train_padded_seqs],y_train,\n",
    "#            batch_size=32,\n",
    "#            epochs=12,\n",
    "#            validation_data=([X_test_padded_seqs, left_test_padded_seqs,right_test_padded_seqs], y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Attention(Layer):\n",
    "    def __init__(self, attention_size, **kwargs):\n",
    "        self.attention_size = attention_size\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    " \n",
    "    def build(self, input_shape):\n",
    "        # W: (EMBED_SIZE, ATTENTION_SIZE)\n",
    "        # b: (ATTENTION_SIZE, 1)\n",
    "        # u: (ATTENTION_SIZE, 1)\n",
    "        self.W = self.add_weight(name=\"W_{:s}\".format(self.name),\n",
    "                                 shape=(input_shape[-1], self.attention_size),\n",
    "                                 initializer=\"glorot_normal\",\n",
    "                                 trainable=True)\n",
    "        self.b = self.add_weight(name=\"b_{:s}\".format(self.name),\n",
    "                                 shape=(input_shape[1], 1),\n",
    "                                 initializer=\"zeros\",\n",
    "                                 trainable=True)\n",
    "        self.u = self.add_weight(name=\"u_{:s}\".format(self.name),\n",
    "                                 shape=(self.attention_size, 1),\n",
    "                                 initializer=\"glorot_normal\",\n",
    "                                 trainable=True)\n",
    "        super(Attention, self).build(input_shape)\n",
    " \n",
    "    def call(self, x, mask=None):\n",
    "        # input: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)\n",
    "        # et: (BATCH_SIZE, MAX_TIMESTEPS, ATTENTION_SIZE)\n",
    "        et = K.tanh(K.dot(x, self.W) + self.b)\n",
    "        # at: (BATCH_SIZE, MAX_TIMESTEPS)\n",
    "        at = K.softmax(K.squeeze(K.dot(et, self.u), axis=-1))\n",
    "        if mask is not None:\n",
    "            at *= K.cast(mask, K.floatx())\n",
    "        # ot: (BATCH_SIZE, MAX_TIMESTEPS, EMBED_SIZE)\n",
    "        atx = K.expand_dims(at, axis=-1)\n",
    "        ot = atx * x\n",
    "        # output: (BATCH_SIZE, EMBED_SIZE)\n",
    "        output = K.sum(ot, axis=1)\n",
    "        return output\n",
    " \n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    " \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], input_shape[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 需要导入两个模型，分别是句子级别的和篇章级别的，以及预处理后的文本序列\n",
    "def get_attention(sent_model, doc_model, sequences, topN=5):\n",
    "    sent_before_att = K.function([sent_model.layers[0].input, K.learning_phase()],\n",
    "                                 [sent_model.layers[2].output])\n",
    "    cnt_reviews = sequences.shape[0]\n",
    " \n",
    "    # 导出这个句子每个词的权重\n",
    "    sent_att_w = sent_model.layers[3].get_weights()\n",
    "    sent_all_att = []\n",
    "    for i in range(cnt_reviews):\n",
    "        sent_each_att = sent_before_att([sequences[i], 0])\n",
    "        sent_each_att = cal_att_weights(sent_each_att, sent_att_w, model_name)\n",
    "        sent_each_att = sent_each_att.ravel()\n",
    "        sent_all_att.append(sent_each_att)\n",
    "    sent_all_att = np.array(sent_all_att)\n",
    " \n",
    "    doc_before_att = K.function([doc_model.layers[0].input, K.learning_phase()],\n",
    "                                [doc_model.layers[2].output])\n",
    "    # 找到重要的分句\n",
    "    doc_att_w = doc_model.layers[3].get_weights()\n",
    "    doc_sub_att = doc_before_att([sequences, 0])\n",
    "    doc_att = cal_att_weights(doc_sub_att, doc_att_w, model_name)\n",
    " \n",
    "    return sent_all_att, doc_att\n",
    " \n",
    "# 使用numpy重新计算attention层的结果\n",
    "def cal_att_weights(output, att_w, model_name):\n",
    "    if model_name == 'HAN':\n",
    "        eij = np.tanh(np.dot(output[0], att_w[0]) + att_w[1])\n",
    "        eij = np.dot(eij, att_w[2])\n",
    "        eij = eij.reshape((eij.shape[0], eij.shape[1]))\n",
    "        ai = np.exp(eij)\n",
    "        weights = ai / np.sum(ai)\n",
    "        return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'AttLayer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-c5cbfa132538>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0membed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mEmbedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m300\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_length\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mgru\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBidirectional\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mGRU\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdropout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0membed\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mattention\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAttLayer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgru\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'softmax'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattention\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mModel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'AttLayer' is not defined"
     ]
    }
   ],
   "source": [
    "# 模型结构：词嵌入-双向GRU-Attention-全连接\n",
    "inputs = Input(shape=(64,), dtype='float64')\n",
    "embed = Embedding(len(vocab) + 1,300, input_length = 64)(inputs)\n",
    "gru = Bidirectional(GRU(100, dropout=0.2, return_sequences=True))(embed)\n",
    "attention = AttLayer()(gru)\n",
    "output = Dense(3, activation='softmax')(attention)\n",
    "model = Model(inputs, output)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_4 (Embedding)      (None, 64, 300)           87423600  \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 3)                 903       \n",
      "=================================================================\n",
      "Total params: 87,424,503\n",
      "Trainable params: 87,424,503\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 模型结构：词嵌入(n-gram)-最大化池化-全连接\n",
    "# 生成n-gram组合的词(以3为例)\n",
    "ngram = 3\n",
    "# 将n-gram词加入到词表\n",
    "def create_ngram(sent, ngram_value):\n",
    "    return set(zip(*[sent[i:] for i in range(ngram_value)]))\n",
    "ngram_set = set()\n",
    "for sentence in x_train_padded_seqs:\n",
    "    for i in range(2, ngram+1):\n",
    "        set_of_ngram = create_ngram(sentence, i)\n",
    "        ngram_set.update(set_of_ngram)\n",
    "        \n",
    "# 给n-gram词汇编码\n",
    "start_index = len(vocab) + 2\n",
    "token_indice = {v: k + start_index for k, v in enumerate(ngram_set)} # 给n-gram词汇编码\n",
    "indice_token = {token_indice[k]: k for k in token_indice}\n",
    "max_features = np.max(list(indice_token.keys())) + 1\n",
    "# 将n-gram词加入到输入文本的末端\n",
    "def add_ngram(sequences, token_indice, ngram_range):\n",
    "    new_sequences = []\n",
    "    for sent in sequences:\n",
    "        new_list = sent[:]\n",
    "        for i in range(len(new_list) - ngram_range + 1):\n",
    "            for ngram_value in range(2, ngram_range + 1):\n",
    "                ngram = tuple(new_list[i:i + ngram_value])\n",
    "                if ngram in token_indice:\n",
    "                    new_list.append(token_indice[ngram])\n",
    "        new_sequences.append(new_list)\n",
    "    return new_sequences\n",
    "  \n",
    "x_train = add_ngram(x_train_word_ids, token_indice, ngram)\n",
    "x_test = add_ngram(x_test_word_ids, token_indice, ngram)\n",
    "x_train = pad_sequences(x_train, maxlen=25)\n",
    "x_test = pad_sequences(x_test, maxlen=25)\n",
    " \n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, 300, input_length=64))\n",
    "model.add(GlobalAveragePooling1D())\n",
    "model.add(Dense(3, activation='softmax'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda\\lib\\site-packages\\tensorflow\\python\\framework\\indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12518/12518 [==============================] - 79s 6ms/step - loss: 1.0832 - accuracy: 0.4070A: 23s - loss: 1.0874 - accuracy:\n",
      "Epoch 2/10\n",
      "12518/12518 [==============================] - 66s 5ms/step - loss: 1.0559 - accuracy: 0.4493\n",
      "Epoch 3/10\n",
      "12518/12518 [==============================] - 61s 5ms/step - loss: 1.0265 - accuracy: 0.5006TA: 58s - loss: 1.0406 - a\n",
      "Epoch 4/10\n",
      "12518/12518 [==============================] - 64s 5ms/step - loss: 0.9956 - accuracy: 0.5141\n",
      "Epoch 5/10\n",
      "12518/12518 [==============================] - 77s 6ms/step - loss: 0.9620 - accuracy: 0.5701\n",
      "Epoch 6/10\n",
      "12518/12518 [==============================] - 66s 5ms/step - loss: 0.9252 - accuracy: 0.5915\n",
      "Epoch 7/10\n",
      "12518/12518 [==============================] - 76s 6ms/step - loss: 0.8844 - accuracy: 0.6394\n",
      "Epoch 8/10\n",
      "12518/12518 [==============================] - 82s 7ms/step - loss: 0.8420 - accuracy: 0.6683\n",
      "Epoch 9/10\n",
      "12518/12518 [==============================] - 94s 7ms/step - loss: 0.8012 - accuracy: 0.6981\n",
      "Epoch 10/10\n",
      "12518/12518 [==============================] - 80s 6ms/step - loss: 0.7634 - accuracy: 0.7183\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x23fc5c89400>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "# one_hot_labels = keras.utils.to_categorical(y_train, num_classes=3)  # 将标签转换为one-hot编码\n",
    "one_hot_labels=y_train\n",
    "model.fit(x_train_padded_seqs, one_hot_labels, batch_size=800, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
